{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from icepyx import icesat2data as ipd\n",
    "import os\n",
    "import shutil\n",
    "from pathlib import Path\n",
    "from pprint import pprint\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download ATL06 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "short_name = 'ATL06'\n",
    "spatial_extent = [ -50.71,  65.73, -45.32,  68.08]\n",
    "date_range = ['2019-06-01','2019-06-05']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ATL06\n",
      "['2019-06-01', '2019-06-05']\n",
      "00:00:00\n",
      "23:59:59\n",
      "003\n",
      "['bounding box', [-50.71, 65.73, -45.32, 68.08]]\n"
     ]
    }
   ],
   "source": [
    "region_a = ipd.Icesat2Data(short_name, spatial_extent, date_range)\n",
    "\n",
    "print(region_a.dataset)\n",
    "print(region_a.dates)\n",
    "print(region_a.start_time)\n",
    "print(region_a.end_time)\n",
    "print(region_a.dataset_version)\n",
    "print(region_a.spatial_extent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ATL06_20190601071608_09780303_003_01.h5',\n",
       " 'ATL06_20190601200102_09860305_003_01.h5',\n",
       " 'ATL06_20190605070748_10390303_003_01.h5',\n",
       " 'ATL06_20190605195242_10470305_003_01.h5']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "region_a.avail_granules(ids=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Earthdata Login password:  ·········\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Invalid username or password, please retry.']\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Please re-enter your Earthdata user ID:  00Alifed!\n",
      "Earthdata Login password:  ·\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Invalid username or password, please retry.']\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Please re-enter your Earthdata user ID:  alicecima\n",
      "Earthdata Login password:  ·········\n"
     ]
    }
   ],
   "source": [
    "earthdata_emails = {'tsnow03':'tasha.snow@colorado.edu',\n",
    "                 'fperez': 'fernando.perez@berkeley.edu',\n",
    "                 'alicecima':'alice_cima@berkeley.edu'\n",
    "                # add your name here\n",
    "                }\n",
    "\n",
    "user = 'tsnow03'\n",
    "region_a.earthdata_login(user, earthdata_emails[user])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "var_list inputs: atlas_sdp_gps_epoch, control, data_end_utc, data_start_utc, end_cycle, end_delta_time, end_geoseg, end_gpssow, end_gpsweek, end_orbit, end_region, end_rgt, granule_end_utc, granule_start_utc, qa_at_interval, release, start_cycle, start_delta_time, start_geoseg, start_gpssow, start_gpsweek, start_orbit, start_region, start_rgt, version, dt_hist, fit_maxiter, fpb_maxiter, maxiter, max_res_ids, min_dist, min_gain_th, min_n_pe, min_n_sel, min_signal_conf, n_hist, nhist_bins, n_sigmas, proc_interval, qs_lim_bsc, qs_lim_hrs, qs_lim_hsigma, qs_lim_msw, qs_lim_snr, qs_lim_sss, rbin_width, sigma_beam, sigma_tx, t_dead, atl06_quality_summary, delta_time, h_li, h_li_sigma, latitude, longitude, segment_id, sigma_geo_h, fpb_mean_corr, fpb_mean_corr_sigma, fpb_med_corr, fpb_med_corr_sigma, fpb_n_corr, med_r_fit, tx_mean_corr, tx_med_corr, dem_flag, dem_h, geoid_h, dh_fit_dx, dh_fit_dx_sigma, dh_fit_dy, h_expected_rms, h_mean, h_rms_misfit, h_robust_sprd, n_fit_photons, n_seg_pulses, sigma_h_mean, signal_selection_source, signal_selection_source_status, snr, snr_significance, w_surface_window_final, bckgrd, bsnow_conf, bsnow_h, bsnow_od, cloud_flg_asr, cloud_flg_atm, dac, e_bckgrd, layer_flag, msw_flag, neutat_delay_total, r_eff, solar_azimuth, solar_elevation, tide_earth, tide_equilibrium, tide_load, tide_ocean, tide_pole, ref_azimuth, ref_coelv, seg_azimuth, sigma_geo_at, sigma_geo_r, sigma_geo_xt, x_atc, y_atc, bckgrd_per_m, bin_top_h, count, ds_segment_id, lat_mean, lon_mean, pulse_count, segment_id_list, x_atc_mean, record_number, reference_pt_lat, reference_pt_lon, signal_selection_status_all, signal_selection_status_backup, signal_selection_status_confident, crossing_time, cycle_number, lan, orbit_number, rgt, sc_orient, sc_orient_time, qa_granule_fail_reason, qa_granule_pass_fail, signal_selection_source_fraction_0, signal_selection_source_fraction_1, signal_selection_source_fraction_2, signal_selection_source_fraction_3\n",
      "keyword_list and beam_list inputs: ancillary_data, bias_correction, dem, fit_statistics, geophysical, ground_track, gt1l, gt1r, gt2l, gt2r, gt3l, gt3r, land_ice, land_ice_segments, none, orbit_info, quality_assessment, residual_histogram, segment_quality, signal_selection_status\n"
     ]
    }
   ],
   "source": [
    "# All possible variables\n",
    "region_a.order_vars.avail(options=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "application/json": {
       "atl06_quality_summary": [
        "gt1l/land_ice_segments/atl06_quality_summary",
        "gt1r/land_ice_segments/atl06_quality_summary",
        "gt2l/land_ice_segments/atl06_quality_summary",
        "gt2r/land_ice_segments/atl06_quality_summary",
        "gt3l/land_ice_segments/atl06_quality_summary",
        "gt3r/land_ice_segments/atl06_quality_summary"
       ],
       "atlas_sdp_gps_epoch": [
        "ancillary_data/atlas_sdp_gps_epoch"
       ],
       "bsnow_conf": [
        "gt1l/land_ice_segments/geophysical/bsnow_conf",
        "gt1r/land_ice_segments/geophysical/bsnow_conf",
        "gt2l/land_ice_segments/geophysical/bsnow_conf",
        "gt2r/land_ice_segments/geophysical/bsnow_conf",
        "gt3l/land_ice_segments/geophysical/bsnow_conf",
        "gt3r/land_ice_segments/geophysical/bsnow_conf"
       ],
       "bsnow_h": [
        "gt1l/land_ice_segments/geophysical/bsnow_h",
        "gt1r/land_ice_segments/geophysical/bsnow_h",
        "gt2l/land_ice_segments/geophysical/bsnow_h",
        "gt2r/land_ice_segments/geophysical/bsnow_h",
        "gt3l/land_ice_segments/geophysical/bsnow_h",
        "gt3r/land_ice_segments/geophysical/bsnow_h"
       ],
       "cloud_flg_asr": [
        "gt1l/land_ice_segments/geophysical/cloud_flg_asr",
        "gt1r/land_ice_segments/geophysical/cloud_flg_asr",
        "gt2l/land_ice_segments/geophysical/cloud_flg_asr",
        "gt2r/land_ice_segments/geophysical/cloud_flg_asr",
        "gt3l/land_ice_segments/geophysical/cloud_flg_asr",
        "gt3r/land_ice_segments/geophysical/cloud_flg_asr"
       ],
       "cloud_flg_atm": [
        "gt1l/land_ice_segments/geophysical/cloud_flg_atm",
        "gt1r/land_ice_segments/geophysical/cloud_flg_atm",
        "gt2l/land_ice_segments/geophysical/cloud_flg_atm",
        "gt2r/land_ice_segments/geophysical/cloud_flg_atm",
        "gt3l/land_ice_segments/geophysical/cloud_flg_atm",
        "gt3r/land_ice_segments/geophysical/cloud_flg_atm"
       ],
       "data_end_utc": [
        "ancillary_data/data_end_utc"
       ],
       "data_start_utc": [
        "ancillary_data/data_start_utc"
       ],
       "delta_time": [
        "gt1l/land_ice_segments/delta_time",
        "gt1l/residual_histogram/delta_time",
        "gt1l/segment_quality/delta_time",
        "gt1r/land_ice_segments/delta_time",
        "gt1r/residual_histogram/delta_time",
        "gt1r/segment_quality/delta_time",
        "gt2l/land_ice_segments/delta_time",
        "gt2l/residual_histogram/delta_time",
        "gt2l/segment_quality/delta_time",
        "gt2r/land_ice_segments/delta_time",
        "gt2r/residual_histogram/delta_time",
        "gt2r/segment_quality/delta_time",
        "gt3l/land_ice_segments/delta_time",
        "gt3l/residual_histogram/delta_time",
        "gt3l/segment_quality/delta_time",
        "gt3r/land_ice_segments/delta_time",
        "gt3r/residual_histogram/delta_time",
        "gt3r/segment_quality/delta_time",
        "quality_assessment/gt1l/delta_time",
        "quality_assessment/gt1r/delta_time",
        "quality_assessment/gt2l/delta_time",
        "quality_assessment/gt2r/delta_time",
        "quality_assessment/gt3l/delta_time",
        "quality_assessment/gt3r/delta_time"
       ],
       "dh_fit_dx": [
        "gt1l/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt1r/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt2l/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt2r/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt3l/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt3r/land_ice_segments/fit_statistics/dh_fit_dx"
       ],
       "dh_fit_dx_sigma": [
        "gt1l/land_ice_segments/fit_statistics/dh_fit_dx_sigma",
        "gt1r/land_ice_segments/fit_statistics/dh_fit_dx_sigma",
        "gt2l/land_ice_segments/fit_statistics/dh_fit_dx_sigma",
        "gt2r/land_ice_segments/fit_statistics/dh_fit_dx_sigma",
        "gt3l/land_ice_segments/fit_statistics/dh_fit_dx_sigma",
        "gt3r/land_ice_segments/fit_statistics/dh_fit_dx_sigma"
       ],
       "dh_fit_dy": [
        "gt1l/land_ice_segments/fit_statistics/dh_fit_dy",
        "gt1r/land_ice_segments/fit_statistics/dh_fit_dy",
        "gt2l/land_ice_segments/fit_statistics/dh_fit_dy",
        "gt2r/land_ice_segments/fit_statistics/dh_fit_dy",
        "gt3l/land_ice_segments/fit_statistics/dh_fit_dy",
        "gt3r/land_ice_segments/fit_statistics/dh_fit_dy"
       ],
       "end_delta_time": [
        "ancillary_data/end_delta_time"
       ],
       "granule_end_utc": [
        "ancillary_data/granule_end_utc"
       ],
       "granule_start_utc": [
        "ancillary_data/granule_start_utc"
       ],
       "h_li": [
        "gt1l/land_ice_segments/h_li",
        "gt1r/land_ice_segments/h_li",
        "gt2l/land_ice_segments/h_li",
        "gt2r/land_ice_segments/h_li",
        "gt3l/land_ice_segments/h_li",
        "gt3r/land_ice_segments/h_li"
       ],
       "h_li_sigma": [
        "gt1l/land_ice_segments/h_li_sigma",
        "gt1r/land_ice_segments/h_li_sigma",
        "gt2l/land_ice_segments/h_li_sigma",
        "gt2r/land_ice_segments/h_li_sigma",
        "gt3l/land_ice_segments/h_li_sigma",
        "gt3r/land_ice_segments/h_li_sigma"
       ],
       "h_mean": [
        "gt1l/land_ice_segments/fit_statistics/h_mean",
        "gt1r/land_ice_segments/fit_statistics/h_mean",
        "gt2l/land_ice_segments/fit_statistics/h_mean",
        "gt2r/land_ice_segments/fit_statistics/h_mean",
        "gt3l/land_ice_segments/fit_statistics/h_mean",
        "gt3r/land_ice_segments/fit_statistics/h_mean"
       ],
       "h_rms_misfit": [
        "gt1l/land_ice_segments/fit_statistics/h_rms_misfit",
        "gt1r/land_ice_segments/fit_statistics/h_rms_misfit",
        "gt2l/land_ice_segments/fit_statistics/h_rms_misfit",
        "gt2r/land_ice_segments/fit_statistics/h_rms_misfit",
        "gt3l/land_ice_segments/fit_statistics/h_rms_misfit",
        "gt3r/land_ice_segments/fit_statistics/h_rms_misfit"
       ],
       "h_robust_sprd": [
        "gt1l/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt1r/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt2l/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt2r/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt3l/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt3r/land_ice_segments/fit_statistics/h_robust_sprd"
       ],
       "latitude": [
        "gt1l/land_ice_segments/latitude",
        "gt1r/land_ice_segments/latitude",
        "gt2l/land_ice_segments/latitude",
        "gt2r/land_ice_segments/latitude",
        "gt3l/land_ice_segments/latitude",
        "gt3r/land_ice_segments/latitude"
       ],
       "longitude": [
        "gt1l/land_ice_segments/longitude",
        "gt1r/land_ice_segments/longitude",
        "gt2l/land_ice_segments/longitude",
        "gt2r/land_ice_segments/longitude",
        "gt3l/land_ice_segments/longitude",
        "gt3r/land_ice_segments/longitude"
       ],
       "n_fit_photons": [
        "gt1l/land_ice_segments/fit_statistics/n_fit_photons",
        "gt1r/land_ice_segments/fit_statistics/n_fit_photons",
        "gt2l/land_ice_segments/fit_statistics/n_fit_photons",
        "gt2r/land_ice_segments/fit_statistics/n_fit_photons",
        "gt3l/land_ice_segments/fit_statistics/n_fit_photons",
        "gt3r/land_ice_segments/fit_statistics/n_fit_photons"
       ],
       "r_eff": [
        "gt1l/land_ice_segments/geophysical/r_eff",
        "gt1r/land_ice_segments/geophysical/r_eff",
        "gt2l/land_ice_segments/geophysical/r_eff",
        "gt2r/land_ice_segments/geophysical/r_eff",
        "gt3l/land_ice_segments/geophysical/r_eff",
        "gt3r/land_ice_segments/geophysical/r_eff"
       ],
       "sc_orient": [
        "orbit_info/sc_orient"
       ],
       "sc_orient_time": [
        "orbit_info/sc_orient_time"
       ],
       "seg_azimuth": [
        "gt1l/land_ice_segments/ground_track/seg_azimuth",
        "gt1r/land_ice_segments/ground_track/seg_azimuth",
        "gt2l/land_ice_segments/ground_track/seg_azimuth",
        "gt2r/land_ice_segments/ground_track/seg_azimuth",
        "gt3l/land_ice_segments/ground_track/seg_azimuth",
        "gt3r/land_ice_segments/ground_track/seg_azimuth"
       ],
       "segment_id": [
        "gt1l/land_ice_segments/segment_id",
        "gt1l/segment_quality/segment_id",
        "gt1r/land_ice_segments/segment_id",
        "gt1r/segment_quality/segment_id",
        "gt2l/land_ice_segments/segment_id",
        "gt2l/segment_quality/segment_id",
        "gt2r/land_ice_segments/segment_id",
        "gt2r/segment_quality/segment_id",
        "gt3l/land_ice_segments/segment_id",
        "gt3l/segment_quality/segment_id",
        "gt3r/land_ice_segments/segment_id",
        "gt3r/segment_quality/segment_id"
       ],
       "sigma_geo_at": [
        "gt1l/land_ice_segments/ground_track/sigma_geo_at",
        "gt1r/land_ice_segments/ground_track/sigma_geo_at",
        "gt2l/land_ice_segments/ground_track/sigma_geo_at",
        "gt2r/land_ice_segments/ground_track/sigma_geo_at",
        "gt3l/land_ice_segments/ground_track/sigma_geo_at",
        "gt3r/land_ice_segments/ground_track/sigma_geo_at"
       ],
       "sigma_geo_h": [
        "gt1l/land_ice_segments/sigma_geo_h",
        "gt1r/land_ice_segments/sigma_geo_h",
        "gt2l/land_ice_segments/sigma_geo_h",
        "gt2r/land_ice_segments/sigma_geo_h",
        "gt3l/land_ice_segments/sigma_geo_h",
        "gt3r/land_ice_segments/sigma_geo_h"
       ],
       "sigma_geo_xt": [
        "gt1l/land_ice_segments/ground_track/sigma_geo_xt",
        "gt1r/land_ice_segments/ground_track/sigma_geo_xt",
        "gt2l/land_ice_segments/ground_track/sigma_geo_xt",
        "gt2r/land_ice_segments/ground_track/sigma_geo_xt",
        "gt3l/land_ice_segments/ground_track/sigma_geo_xt",
        "gt3r/land_ice_segments/ground_track/sigma_geo_xt"
       ],
       "signal_selection_source": [
        "gt1l/land_ice_segments/fit_statistics/signal_selection_source",
        "gt1l/segment_quality/signal_selection_source",
        "gt1r/land_ice_segments/fit_statistics/signal_selection_source",
        "gt1r/segment_quality/signal_selection_source",
        "gt2l/land_ice_segments/fit_statistics/signal_selection_source",
        "gt2l/segment_quality/signal_selection_source",
        "gt2r/land_ice_segments/fit_statistics/signal_selection_source",
        "gt2r/segment_quality/signal_selection_source",
        "gt3l/land_ice_segments/fit_statistics/signal_selection_source",
        "gt3l/segment_quality/signal_selection_source",
        "gt3r/land_ice_segments/fit_statistics/signal_selection_source",
        "gt3r/segment_quality/signal_selection_source"
       ],
       "snr_significance": [
        "gt1l/land_ice_segments/fit_statistics/snr_significance",
        "gt1r/land_ice_segments/fit_statistics/snr_significance",
        "gt2l/land_ice_segments/fit_statistics/snr_significance",
        "gt2r/land_ice_segments/fit_statistics/snr_significance",
        "gt3l/land_ice_segments/fit_statistics/snr_significance",
        "gt3r/land_ice_segments/fit_statistics/snr_significance"
       ],
       "start_delta_time": [
        "ancillary_data/start_delta_time"
       ],
       "tide_ocean": [
        "gt1l/land_ice_segments/geophysical/tide_ocean",
        "gt1r/land_ice_segments/geophysical/tide_ocean",
        "gt2l/land_ice_segments/geophysical/tide_ocean",
        "gt2r/land_ice_segments/geophysical/tide_ocean",
        "gt3l/land_ice_segments/geophysical/tide_ocean",
        "gt3r/land_ice_segments/geophysical/tide_ocean"
       ],
       "w_surface_window_final": [
        "gt1l/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt1r/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt2l/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt2r/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt3l/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt3r/land_ice_segments/fit_statistics/w_surface_window_final"
       ],
       "x_atc": [
        "gt1l/land_ice_segments/ground_track/x_atc",
        "gt1r/land_ice_segments/ground_track/x_atc",
        "gt2l/land_ice_segments/ground_track/x_atc",
        "gt2r/land_ice_segments/ground_track/x_atc",
        "gt3l/land_ice_segments/ground_track/x_atc",
        "gt3r/land_ice_segments/ground_track/x_atc"
       ],
       "y_atc": [
        "gt1l/land_ice_segments/ground_track/y_atc",
        "gt1r/land_ice_segments/ground_track/y_atc",
        "gt2l/land_ice_segments/ground_track/y_atc",
        "gt2r/land_ice_segments/ground_track/y_atc",
        "gt3l/land_ice_segments/ground_track/y_atc",
        "gt3r/land_ice_segments/ground_track/y_atc"
       ]
      },
      "text/plain": [
       "<IPython.core.display.JSON object>"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "application/json": {
       "expanded": false,
       "root": "root"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Choose variables to order\n",
    "# region_a.order_vars.remove(all=True)\n",
    "#region_a.order_vars.append(defaults=True)\n",
    "#from IPython.display import JSON\n",
    "#JSON(region_a.order_vars.wanted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/json": {
       "atl06_quality_summary": [
        "gt1l/land_ice_segments/atl06_quality_summary",
        "gt1r/land_ice_segments/atl06_quality_summary",
        "gt2l/land_ice_segments/atl06_quality_summary",
        "gt2r/land_ice_segments/atl06_quality_summary",
        "gt3l/land_ice_segments/atl06_quality_summary",
        "gt3r/land_ice_segments/atl06_quality_summary"
       ],
       "atlas_sdp_gps_epoch": [
        "ancillary_data/atlas_sdp_gps_epoch"
       ],
       "bckgrd": [
        "gt1l/land_ice_segments/geophysical/bckgrd",
        "gt1r/land_ice_segments/geophysical/bckgrd",
        "gt2l/land_ice_segments/geophysical/bckgrd",
        "gt2r/land_ice_segments/geophysical/bckgrd",
        "gt3l/land_ice_segments/geophysical/bckgrd",
        "gt3r/land_ice_segments/geophysical/bckgrd"
       ],
       "bsnow_conf": [
        "gt1l/land_ice_segments/geophysical/bsnow_conf",
        "gt1r/land_ice_segments/geophysical/bsnow_conf",
        "gt2l/land_ice_segments/geophysical/bsnow_conf",
        "gt2r/land_ice_segments/geophysical/bsnow_conf",
        "gt3l/land_ice_segments/geophysical/bsnow_conf",
        "gt3r/land_ice_segments/geophysical/bsnow_conf"
       ],
       "bsnow_h": [
        "gt1l/land_ice_segments/geophysical/bsnow_h",
        "gt1r/land_ice_segments/geophysical/bsnow_h",
        "gt2l/land_ice_segments/geophysical/bsnow_h",
        "gt2r/land_ice_segments/geophysical/bsnow_h",
        "gt3l/land_ice_segments/geophysical/bsnow_h",
        "gt3r/land_ice_segments/geophysical/bsnow_h"
       ],
       "bsnow_od": [
        "gt1l/land_ice_segments/geophysical/bsnow_od",
        "gt1r/land_ice_segments/geophysical/bsnow_od",
        "gt2l/land_ice_segments/geophysical/bsnow_od",
        "gt2r/land_ice_segments/geophysical/bsnow_od",
        "gt3l/land_ice_segments/geophysical/bsnow_od",
        "gt3r/land_ice_segments/geophysical/bsnow_od"
       ],
       "cloud_flg_asr": [
        "gt1l/land_ice_segments/geophysical/cloud_flg_asr",
        "gt1r/land_ice_segments/geophysical/cloud_flg_asr",
        "gt2l/land_ice_segments/geophysical/cloud_flg_asr",
        "gt2r/land_ice_segments/geophysical/cloud_flg_asr",
        "gt3l/land_ice_segments/geophysical/cloud_flg_asr",
        "gt3r/land_ice_segments/geophysical/cloud_flg_asr"
       ],
       "cloud_flg_atm": [
        "gt1l/land_ice_segments/geophysical/cloud_flg_atm",
        "gt1r/land_ice_segments/geophysical/cloud_flg_atm",
        "gt2l/land_ice_segments/geophysical/cloud_flg_atm",
        "gt2r/land_ice_segments/geophysical/cloud_flg_atm",
        "gt3l/land_ice_segments/geophysical/cloud_flg_atm",
        "gt3r/land_ice_segments/geophysical/cloud_flg_atm"
       ],
       "data_end_utc": [
        "ancillary_data/data_end_utc"
       ],
       "data_start_utc": [
        "ancillary_data/data_start_utc"
       ],
       "dh_fit_dx": [
        "gt1l/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt1r/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt2l/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt2r/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt3l/land_ice_segments/fit_statistics/dh_fit_dx",
        "gt3r/land_ice_segments/fit_statistics/dh_fit_dx"
       ],
       "e_bckgrd": [
        "gt1l/land_ice_segments/geophysical/e_bckgrd",
        "gt1r/land_ice_segments/geophysical/e_bckgrd",
        "gt2l/land_ice_segments/geophysical/e_bckgrd",
        "gt2r/land_ice_segments/geophysical/e_bckgrd",
        "gt3l/land_ice_segments/geophysical/e_bckgrd",
        "gt3r/land_ice_segments/geophysical/e_bckgrd"
       ],
       "end_delta_time": [
        "ancillary_data/end_delta_time"
       ],
       "end_geoseg": [
        "ancillary_data/end_geoseg"
       ],
       "granule_end_utc": [
        "ancillary_data/granule_end_utc"
       ],
       "granule_start_utc": [
        "ancillary_data/granule_start_utc"
       ],
       "h_li": [
        "gt1l/land_ice_segments/h_li",
        "gt1r/land_ice_segments/h_li",
        "gt2l/land_ice_segments/h_li",
        "gt2r/land_ice_segments/h_li",
        "gt3l/land_ice_segments/h_li",
        "gt3r/land_ice_segments/h_li"
       ],
       "h_li_sigma": [
        "gt1l/land_ice_segments/h_li_sigma",
        "gt1r/land_ice_segments/h_li_sigma",
        "gt2l/land_ice_segments/h_li_sigma",
        "gt2r/land_ice_segments/h_li_sigma",
        "gt3l/land_ice_segments/h_li_sigma",
        "gt3r/land_ice_segments/h_li_sigma"
       ],
       "h_robust_sprd": [
        "gt1l/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt1r/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt2l/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt2r/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt3l/land_ice_segments/fit_statistics/h_robust_sprd",
        "gt3r/land_ice_segments/fit_statistics/h_robust_sprd"
       ],
       "latitude": [
        "gt1l/land_ice_segments/latitude",
        "gt1r/land_ice_segments/latitude",
        "gt2l/land_ice_segments/latitude",
        "gt2r/land_ice_segments/latitude",
        "gt3l/land_ice_segments/latitude",
        "gt3r/land_ice_segments/latitude"
       ],
       "layer_flag": [
        "gt1l/land_ice_segments/geophysical/layer_flag",
        "gt1r/land_ice_segments/geophysical/layer_flag",
        "gt2l/land_ice_segments/geophysical/layer_flag",
        "gt2r/land_ice_segments/geophysical/layer_flag",
        "gt3l/land_ice_segments/geophysical/layer_flag",
        "gt3r/land_ice_segments/geophysical/layer_flag"
       ],
       "longitude": [
        "gt1l/land_ice_segments/longitude",
        "gt1r/land_ice_segments/longitude",
        "gt2l/land_ice_segments/longitude",
        "gt2r/land_ice_segments/longitude",
        "gt3l/land_ice_segments/longitude",
        "gt3r/land_ice_segments/longitude"
       ],
       "msw_flag": [
        "gt1l/land_ice_segments/geophysical/msw_flag",
        "gt1r/land_ice_segments/geophysical/msw_flag",
        "gt2l/land_ice_segments/geophysical/msw_flag",
        "gt2r/land_ice_segments/geophysical/msw_flag",
        "gt3l/land_ice_segments/geophysical/msw_flag",
        "gt3r/land_ice_segments/geophysical/msw_flag"
       ],
       "n_fit_photons": [
        "gt1l/land_ice_segments/fit_statistics/n_fit_photons",
        "gt1r/land_ice_segments/fit_statistics/n_fit_photons",
        "gt2l/land_ice_segments/fit_statistics/n_fit_photons",
        "gt2r/land_ice_segments/fit_statistics/n_fit_photons",
        "gt3l/land_ice_segments/fit_statistics/n_fit_photons",
        "gt3r/land_ice_segments/fit_statistics/n_fit_photons"
       ],
       "sc_orient": [
        "orbit_info/sc_orient"
       ],
       "sc_orient_time": [
        "orbit_info/sc_orient_time"
       ],
       "segment_id": [
        "gt1l/land_ice_segments/segment_id",
        "gt1l/segment_quality/segment_id",
        "gt1r/land_ice_segments/segment_id",
        "gt1r/segment_quality/segment_id",
        "gt2l/land_ice_segments/segment_id",
        "gt2l/segment_quality/segment_id",
        "gt2r/land_ice_segments/segment_id",
        "gt2r/segment_quality/segment_id",
        "gt3l/land_ice_segments/segment_id",
        "gt3l/segment_quality/segment_id",
        "gt3r/land_ice_segments/segment_id",
        "gt3r/segment_quality/segment_id"
       ],
       "signal_selection_source": [
        "gt1l/land_ice_segments/fit_statistics/signal_selection_source",
        "gt1l/segment_quality/signal_selection_source",
        "gt1r/land_ice_segments/fit_statistics/signal_selection_source",
        "gt1r/segment_quality/signal_selection_source",
        "gt2l/land_ice_segments/fit_statistics/signal_selection_source",
        "gt2l/segment_quality/signal_selection_source",
        "gt2r/land_ice_segments/fit_statistics/signal_selection_source",
        "gt2r/segment_quality/signal_selection_source",
        "gt3l/land_ice_segments/fit_statistics/signal_selection_source",
        "gt3l/segment_quality/signal_selection_source",
        "gt3r/land_ice_segments/fit_statistics/signal_selection_source",
        "gt3r/segment_quality/signal_selection_source"
       ],
       "snr_significance": [
        "gt1l/land_ice_segments/fit_statistics/snr_significance",
        "gt1r/land_ice_segments/fit_statistics/snr_significance",
        "gt2l/land_ice_segments/fit_statistics/snr_significance",
        "gt2r/land_ice_segments/fit_statistics/snr_significance",
        "gt3l/land_ice_segments/fit_statistics/snr_significance",
        "gt3r/land_ice_segments/fit_statistics/snr_significance"
       ],
       "start_delta_time": [
        "ancillary_data/start_delta_time"
       ],
       "w_surface_window_final": [
        "gt1l/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt1r/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt2l/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt2r/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt3l/land_ice_segments/fit_statistics/w_surface_window_final",
        "gt3r/land_ice_segments/fit_statistics/w_surface_window_final"
       ]
      },
      "text/plain": [
       "<IPython.core.display.JSON object>"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "application/json": {
       "expanded": false,
       "root": "root"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "region_a.order_vars.append(var_list=['latitude','longitude','h_li','h_li_sigma','atl06_quality_summary',\n",
    "                                      'signal_selection_source','snr_significance','h_robust_sprd','dh_fit_dx','bsnow_conf',\n",
    "                                      'cloud_flg_asr','cloud_flg_atm','msw_flag','bsnow_h','bsnow_od','layer_flag','bckgrd',\n",
    "                                      'e_bckgrd','n_fit_photons','end_geoseg','segment_id','w_surface_window_final'])\n",
    "from IPython.display import JSON\n",
    "JSON(region_a.order_vars.wanted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, specify our coverage request with the wanted variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "region_a.subsetparams(Coverage=region_a.order_vars.wanted);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of data order requests is  1  for  4  granules.\n",
      "Data request  1  of  1  is submitting to NSIDC\n",
      "order ID:  5000000701884\n",
      "Initial status of your order request at NSIDC is:  processing\n",
      "Your order status is still  processing  at NSIDC. Please continue waiting... this may take a few moments.\n",
      "Your order is: complete\n"
     ]
    }
   ],
   "source": [
    "region_a.order_granules()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['5000000701884']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View a short list of order IDs:\n",
    "region_a.granules.orderIDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beginning download of zipped output...\n",
      "Data request 5000000701884 of  1  order(s) is downloaded.\n",
      "Download complete\n"
     ]
    }
   ],
   "source": [
    "path = './download'\n",
    "# without variable subsetting, or with variable subsetting if you have run region_a.order_granules(Coverage=region_a.order_vars.wanted)\n",
    "region_a.download_granules(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.5M\t./download\n",
      "9.5M\ttotal\n"
     ]
    }
   ],
   "source": [
    "!du -csh ./download"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Pandas DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyproj\n",
    "from astropy.time import Time\n",
    "\n",
    "def gps2dyr(time):\n",
    "    \"\"\"Converte GPS time to decimal years.\"\"\"\n",
    "    return Time(time, format='gps').decimalyear\n",
    "\n",
    "\n",
    "def orbit_type(time, lat, tmax=1):\n",
    "    \"\"\"Separate tracks into ascending and descending.\n",
    "    \n",
    "    Defines tracks as segments with time breaks > tmax,\n",
    "    and tests whether lat increases or decreases w/time.\n",
    "    \"\"\"\n",
    "    tracks = np.zeros(lat.shape)  # generate track segment\n",
    "    tracks[0:np.argmax(np.abs(lat))] = 1  # set values for segment\n",
    "    is_asc = np.zeros(tracks.shape, dtype=bool)  # output index array\n",
    "\n",
    "    # Loop trough individual secments\n",
    "    for track in np.unique(tracks):\n",
    "    \n",
    "        i_track, = np.where(track == tracks)  # get all pts from seg\n",
    "    \n",
    "        if len(i_track) < 2: continue\n",
    "    \n",
    "        # Test if lat increases (asc) or decreases (des) w/time\n",
    "        i_min = time[i_track].argmin()\n",
    "        i_max = time[i_track].argmax()\n",
    "        lat_diff = lat[i_track][i_max] - lat[i_track][i_min]\n",
    "    \n",
    "        # Determine track type\n",
    "        if lat_diff > 0:  is_asc[i_track] = True\n",
    "    \n",
    "    return is_asc\n",
    "\n",
    "\n",
    "def transform_coord(proj1, proj2, x, y):\n",
    "    \"\"\"Transform coordinates from proj1 to proj2 (EPSG num).\n",
    "\n",
    "    Example EPSG projections:\n",
    "        Geodetic (lon/lat): 4326\n",
    "        Polar Stereo AnIS (x/y): 3031\n",
    "        Polar Stereo GrIS (x/y): 3413\n",
    "    \"\"\"\n",
    "    # Set full EPSG projection strings\n",
    "    proj1 = pyproj.Proj(\"+init=EPSG:\"+str(proj1))\n",
    "    proj2 = pyproj.Proj(\"+init=EPSG:\"+str(proj2))\n",
    "    return pyproj.transform(proj1, proj2, x, y)  # convert\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "import numpy as np\n",
    "\n",
    "def read_atl06(fname, outdir='data', bbox=None):\n",
    "    \"\"\"Read one ATL06 file and output 6 reduced files. \n",
    "    \n",
    "    Extract variables of interest and separate the ATL06 file \n",
    "    into each beam (ground track) and ascending/descending orbits.\n",
    "    \"\"\"\n",
    "\n",
    "    # Each beam is a group\n",
    "    group = ['/gt1l', '/gt1r', '/gt2l', '/gt2r', '/gt3l', '/gt3r']\n",
    "    \n",
    "    # Loop trough beams\n",
    "    dataframes = []  # one dataframe per track\n",
    "    \n",
    "    with h5py.File(fname, 'r') as fi:\n",
    "        # Check which ground tracks are present in this file\n",
    "        gtracks = sorted(['/'+k for k in fi.keys() if k.startswith('gt')])\n",
    "        print('Available ground tracks:', gtracks)\n",
    "    \n",
    "        for k, g in enumerate(gtracks): \n",
    "            # Read in data for a single beam\n",
    "            data = {}\n",
    "            data['ground_track'] = None # Put it first in the dict for column ordering\n",
    "    \n",
    "            # Load vars into memory (include as many as you want)\n",
    "            data['lat'] = fi[g+'/land_ice_segments/latitude'][:]\n",
    "            data['lon'] = fi[g+'/land_ice_segments/longitude'][:]\n",
    "            data['h_li'] = fi[g+'/land_ice_segments/h_li'][:]\n",
    "            data['s_li'] = fi[g+'/land_ice_segments/h_li_sigma'][:]\n",
    "            data['q_flag'] = fi[g+'/land_ice_segments/atl06_quality_summary'][:]\n",
    "            data['s_fg'] = fi[g+'/land_ice_segments/fit_statistics/signal_selection_source'][:]\n",
    "            data['snr'] = fi[g+'/land_ice_segments/fit_statistics/snr_significance'][:]\n",
    "            data['h_rb'] = fi[g+'/land_ice_segments/fit_statistics/h_robust_sprd'][:]\n",
    "            data['dh_fit_dx'] = fi[g+'/land_ice_segments/fit_statistics/dh_fit_dx'][:]\n",
    "            data['b_snow_conf'] = fi[g+'/land_ice_segments/geophysical/bsnow_conf'][:]\n",
    "            \n",
    "            data['c_flg_asr'] = fi[g+'/land_ice_segments/geophysical/cloud_flg_asr'][:]\n",
    "            data['c_flg_atm'] = fi[g+'/land_ice_segments/geophysical/cloud_flg_atm'][:]\n",
    "            data['msw'] = fi[g+'/land_ice_segments/geophysical/msw_flag'][:]\n",
    "            data['bsnow_h'] = fi[g+'/land_ice_segments/geophysical/bsnow_h'][:]\n",
    "            data['bsnow_od'] = fi[g+'/land_ice_segments/geophysical/bsnow_od'][:]\n",
    "            data['layer_flag'] = fi[g+'/land_ice_segments/geophysical/layer_flag'][:]\n",
    "            data['bckgrd'] = fi[g+'/land_ice_segments/geophysical/bckgrd'][:]\n",
    "            data['e_bckgrd'] = fi[g+'/land_ice_segments/geophysical/e_bckgrd'][:]\n",
    "            data['n_fit_photons'] = fi[g+'/land_ice_segments/fit_statistics/n_fit_photons'][:]\n",
    "            #data['end_geoseg'] = fi['/ancillary_data/end_geoseg'][:]\n",
    "            data['segment_id'] = fi[g+'/segment_quality/segment_id'][:]\n",
    "            data['w_surface_window_final'] = fi[g+'/land_ice_segments/fit_statistics/w_surface_window_final'][:]\n",
    "\n",
    "            '''\n",
    "            ***Continue adding columns for these vars, repeated as above. Need to find group/file \n",
    "               structure for each in https://nsidc.org/sites/nsidc.org/files/technical-references/ICESat2_ATL06_data_dict_v003.pdf):\n",
    "               cloud_flg_asr, cloud_flg_atm, msw_flag, bsnow_h, bsnow_od, layer_flag, bckgrd, \n",
    "               e_bckgrd, n_fit_photons, end_geoseg, segment_id, w_surface_window_final\n",
    "            ''' \n",
    "            \n",
    "#             data['dac'] = fi[g+'/land_ice_segments/geophysical/dac'][:]\n",
    "#             data['t_dt'] = fi[g+'/land_ice_segments/delta_time'][:]\n",
    "#             data['tide_earth'] = fi[g+'/land_ice_segments/geophysical/tide_earth'][:]\n",
    "#             data['tide_load'] = fi[g+'/land_ice_segments/geophysical/tide_load'][:]\n",
    "#             data['tide_ocean'] = fi[g+'/land_ice_segments/geophysical/tide_ocean'][:]\n",
    "#             data['tide_pole'] = fi[g+'/land_ice_segments/geophysical/tide_pole'][:]\n",
    "                \n",
    "#                rgt = fi['/orbit_info/rgt'][:]                           # single value\n",
    "#                 t_ref = fi['/ancillary_data/atlas_sdp_gps_epoch'][:]     # single value\n",
    "#                 beam_type = fi[g].attrs[\"atlas_beam_type\"].decode()      # strong/weak (str)\n",
    "#                 spot_number = fi[g].attrs[\"atlas_spot_number\"].decode()  # number (str)\n",
    "        \n",
    "            # Assume all vector fields are of the length of the first. TODO - check?\n",
    "            npts = len(data['lat'])\n",
    "            data['ground_track'] = [g[1:]]*npts\n",
    "                    \n",
    "            # Make a dataframe out of our data dict and store it.\n",
    "            dataframes.append(pd.DataFrame.from_dict(data))\n",
    "        \n",
    "    return dataframes\n",
    "\n",
    "def atl06_2_df(files):\n",
    "    \"\"\"Return a single Pandas dataframe from a list of HDF5 ATL-06 data files.\n",
    "    \"\"\"\n",
    "    dataframes = []\n",
    "    for f in files:\n",
    "        dataframes.extend(read_atl06(f))\n",
    "    \n",
    "    ndfs = len(dataframes)\n",
    "    i = 0\n",
    "    \n",
    "    # pd.concat can only work with up to 10 dataframes at a time,\n",
    "    # so we need to chunk this up\n",
    "    new_dfs = []\n",
    "    while i <= ndfs:\n",
    "        i_end = i+10 if i+10 < ndfs else ndfs\n",
    "        dfs = dataframes[i:i_end]\n",
    "        if not dfs:\n",
    "            break\n",
    "        new_dfs.append(pd.concat(dfs))\n",
    "        i = i_end\n",
    "    \n",
    "    return pd.concat(new_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available ground tracks: ['/gt1l', '/gt1r', '/gt2l', '/gt2r', '/gt3l', '/gt3r']\n",
      "Available ground tracks: ['/gt1l', '/gt1r', '/gt2l', '/gt2r', '/gt3l', '/gt3r']\n",
      "Available ground tracks: ['/gt1l', '/gt1r', '/gt2l', '/gt2r', '/gt3l', '/gt3r']\n",
      "Available ground tracks: ['/gt1l', '/gt1r', '/gt2l', '/gt2r', '/gt3l', '/gt3r']\n"
     ]
    }
   ],
   "source": [
    "path = Path('download')\n",
    "dataf = atl06_2_df(path.glob('*.h5'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ground_track', 'lat', 'lon', 'h_li', 's_li', 'q_flag', 's_fg', 'snr',\n",
       "       'h_rb', 'dh_fit_dx', 'b_snow_conf', 'c_flg_asr', 'c_flg_atm', 'msw',\n",
       "       'bsnow_h', 'bsnow_od', 'layer_flag', 'bckgrd', 'e_bckgrd',\n",
       "       'n_fit_photons', 'segment_id', 'w_surface_window_final'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataf.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataf.to_csv('atl06.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>h_li</th>\n",
       "      <th>s_li</th>\n",
       "      <th>q_flag</th>\n",
       "      <th>s_fg</th>\n",
       "      <th>snr</th>\n",
       "      <th>h_rb</th>\n",
       "      <th>dh_fit_dx</th>\n",
       "      <th>b_snow_conf</th>\n",
       "      <th>...</th>\n",
       "      <th>c_flg_atm</th>\n",
       "      <th>msw</th>\n",
       "      <th>bsnow_h</th>\n",
       "      <th>bsnow_od</th>\n",
       "      <th>layer_flag</th>\n",
       "      <th>bckgrd</th>\n",
       "      <th>e_bckgrd</th>\n",
       "      <th>n_fit_photons</th>\n",
       "      <th>segment_id</th>\n",
       "      <th>w_surface_window_final</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>252416.000000</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>2.524160e+05</td>\n",
       "      <td>252416.000000</td>\n",
       "      <td>2.524160e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>66.969175</td>\n",
       "      <td>-47.371667</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>0.022479</td>\n",
       "      <td>0.037941</td>\n",
       "      <td>0.004300</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>-0.776266</td>\n",
       "      <td>...</td>\n",
       "      <td>0.308368</td>\n",
       "      <td>0.975453</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>0.084068</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>1.155374e+07</td>\n",
       "      <td>529968.532605</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.672515</td>\n",
       "      <td>1.208278</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>0.148235</td>\n",
       "      <td>0.291183</td>\n",
       "      <td>0.050107</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>1.771065</td>\n",
       "      <td>...</td>\n",
       "      <td>0.481262</td>\n",
       "      <td>1.418638</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>0.277490</td>\n",
       "      <td>inf</td>\n",
       "      <td>inf</td>\n",
       "      <td>1.570908e+08</td>\n",
       "      <td>125078.188503</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>65.730013</td>\n",
       "      <td>-49.075188</td>\n",
       "      <td>1.244566e+03</td>\n",
       "      <td>3.356113e-03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.438600e-03</td>\n",
       "      <td>-7.799568e-01</td>\n",
       "      <td>-3.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.995849e+01</td>\n",
       "      <td>9.251221e-03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.070931e+06</td>\n",
       "      <td>5.854139e+05</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>365595.000000</td>\n",
       "      <td>3.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>66.391515</td>\n",
       "      <td>-48.461860</td>\n",
       "      <td>1.483578e+03</td>\n",
       "      <td>1.438970e-02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.277712e-01</td>\n",
       "      <td>-4.686356e-03</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.368811e+06</td>\n",
       "      <td>7.844775e+05</td>\n",
       "      <td>1.150000e+02</td>\n",
       "      <td>375133.000000</td>\n",
       "      <td>3.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>67.015728</td>\n",
       "      <td>-47.793800</td>\n",
       "      <td>1.679594e+03</td>\n",
       "      <td>2.080464e-02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.596234e-01</td>\n",
       "      <td>-8.364916e-04</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.230842e+06</td>\n",
       "      <td>2.222772e+06</td>\n",
       "      <td>1.360000e+02</td>\n",
       "      <td>625393.000000</td>\n",
       "      <td>3.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>67.548462</td>\n",
       "      <td>-46.083928</td>\n",
       "      <td>2.108787e+03</td>\n",
       "      <td>3.137868e-02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.908423e-01</td>\n",
       "      <td>3.726661e-03</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.223469e+06</td>\n",
       "      <td>2.496404e+06</td>\n",
       "      <td>4.380000e+02</td>\n",
       "      <td>630652.000000</td>\n",
       "      <td>3.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>68.079999</td>\n",
       "      <td>-45.320004</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.941731</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>3.402823e+38</td>\n",
       "      <td>2.147484e+09</td>\n",
       "      <td>636233.000000</td>\n",
       "      <td>3.402823e+38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 lat            lon          h_li          s_li  \\\n",
       "count  252416.000000  252416.000000  2.524160e+05  2.524160e+05   \n",
       "mean       66.969175     -47.371667           inf           inf   \n",
       "std         0.672515       1.208278           inf           inf   \n",
       "min        65.730013     -49.075188  1.244566e+03  3.356113e-03   \n",
       "25%        66.391515     -48.461860  1.483578e+03  1.438970e-02   \n",
       "50%        67.015728     -47.793800  1.679594e+03  2.080464e-02   \n",
       "75%        67.548462     -46.083928  2.108787e+03  3.137868e-02   \n",
       "max        68.079999     -45.320004  3.402823e+38  3.402823e+38   \n",
       "\n",
       "              q_flag           s_fg            snr          h_rb  \\\n",
       "count  252416.000000  252416.000000  252416.000000  2.524160e+05   \n",
       "mean        0.022479       0.037941       0.004300           inf   \n",
       "std         0.148235       0.291183       0.050107           inf   \n",
       "min         0.000000       0.000000       0.000000  3.438600e-03   \n",
       "25%         0.000000       0.000000       0.000000  1.277712e-01   \n",
       "50%         0.000000       0.000000       0.000000  1.596234e-01   \n",
       "75%         0.000000       0.000000       0.000000  1.908423e-01   \n",
       "max         1.000000       3.000000       0.941731  3.402823e+38   \n",
       "\n",
       "          dh_fit_dx    b_snow_conf  ...      c_flg_atm            msw  \\\n",
       "count  2.524160e+05  252416.000000  ...  252416.000000  252416.000000   \n",
       "mean            inf      -0.776266  ...       0.308368       0.975453   \n",
       "std             inf       1.771065  ...       0.481262       1.418638   \n",
       "min   -7.799568e-01      -3.000000  ...       0.000000       0.000000   \n",
       "25%   -4.686356e-03      -2.000000  ...       0.000000       0.000000   \n",
       "50%   -8.364916e-04      -1.000000  ...       0.000000       0.000000   \n",
       "75%    3.726661e-03      -1.000000  ...       1.000000       2.000000   \n",
       "max    3.402823e+38       6.000000  ...       2.000000       4.000000   \n",
       "\n",
       "            bsnow_h      bsnow_od     layer_flag        bckgrd      e_bckgrd  \\\n",
       "count  2.524160e+05  2.524160e+05  252416.000000  2.524160e+05  2.524160e+05   \n",
       "mean            inf           inf       0.084068           inf           inf   \n",
       "std             inf           inf       0.277490           inf           inf   \n",
       "min    5.995849e+01  9.251221e-03       0.000000  1.070931e+06  5.854139e+05   \n",
       "25%    3.402823e+38  3.402823e+38       0.000000  2.368811e+06  7.844775e+05   \n",
       "50%    3.402823e+38  3.402823e+38       0.000000  6.230842e+06  2.222772e+06   \n",
       "75%    3.402823e+38  3.402823e+38       0.000000  7.223469e+06  2.496404e+06   \n",
       "max    3.402823e+38  3.402823e+38       1.000000  3.402823e+38  3.402823e+38   \n",
       "\n",
       "       n_fit_photons     segment_id  w_surface_window_final  \n",
       "count   2.524160e+05  252416.000000            2.524160e+05  \n",
       "mean    1.155374e+07  529968.532605                     inf  \n",
       "std     1.570908e+08  125078.188503                     inf  \n",
       "min     3.000000e+00  365595.000000            3.000000e+00  \n",
       "25%     1.150000e+02  375133.000000            3.000000e+00  \n",
       "50%     1.360000e+02  625393.000000            3.000000e+00  \n",
       "75%     4.380000e+02  630652.000000            3.000000e+00  \n",
       "max     2.147484e+09  636233.000000            3.402823e+38  \n",
       "\n",
       "[8 rows x 21 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataf.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    246742\n",
       "1      5674\n",
       "Name: q_flag, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataf.q_flag.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1043409 entries, 0 to 37\n",
      "Data columns (total 11 columns):\n",
      " #   Column        Non-Null Count    Dtype  \n",
      "---  ------        --------------    -----  \n",
      " 0   ground_track  1043409 non-null  object \n",
      " 1   lat           1043409 non-null  float64\n",
      " 2   lon           1043409 non-null  float64\n",
      " 3   h_li          1043409 non-null  float32\n",
      " 4   s_li          1043409 non-null  float32\n",
      " 5   q_flag        1043409 non-null  int8   \n",
      " 6   s_fg          1043409 non-null  int8   \n",
      " 7   snr           1043409 non-null  float32\n",
      " 8   h_rb          1043409 non-null  float32\n",
      " 9   dh_fit_dx     1043409 non-null  float32\n",
      " 10  f_sn          1043409 non-null  float32\n",
      "dtypes: float32(6), float64(2), int8(2), object(1)\n",
      "memory usage: 57.7+ MB\n"
     ]
    }
   ],
   "source": [
    "dataf.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:notebook] *",
   "language": "python",
   "name": "conda-env-notebook-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
